#!/data/Software/mydan/perl/bin/perl -I/data/Software/mydan/JOBX/lib -I/data/Software/mydan/JOBX/private/lib
use strict;
use warnings;
use FindBin qw( $RealBin );
use Data::Dumper;
use POSIX;
use JSON;
use Util;
use Logs;
use LWP::UserAgent;
use MYDan::Collector::Util;

$| ++;

=head1 SYNOPSIS

    db => $mysql,

=cut

my %c = (
  job_api => 5,
  job_server_approval => 1,
  job_server_bury => 1,
  job_server_call => 1,
  job_server_clean => 1,
  job_server_crontab => 1,
  job_server_keepalive => 1,
  job_server_notify => 1,
  job_supervisor => 17,

  jobx_api => 5,
  jobx_server_bury => 1,
  jobx_server_call => 1,
  jobx_server_keepalive => 1,
  jobx_server_monitor => 1,
  jobx_supervisor => 14,

  agent_api => 5,
  agent_server_bury => 1,
  agent_server_call => 1,
  agent_server_check => 1,
  agent_server_inherit => 1,
  agent_server_keepalive => 1,
  agent_supervisor => 15,

  ci_api => 5,
  ci_server_build => 1,
  ci_server_bury => 1,
  ci_server_clean => 1,
  ci_server_findtags => 1,
  ci_server_keepalive => 1,
  ci_server_pkgrsync => 1,
  ci_supervisor => 16,

  connector_api => 10,
  connector_supervisor => 10,

  'job_webshell' => 1,
);


return sub
{
    my %param = @_;

    my ( $myname, $db, $logs ) = ( Util::myname(), $param{db}, Logs->new( 'monitor' ) );
    my $slave = eval{ $db->query( sprintf "select slave from keepalive" ); };
    $logs->die( "get slave from keepalive fail: $@" ) if $@;

    my %slave = map{ $_->[0] => 1 }@$slave;;
    my $slavepath = "/data/glusterfs/slavemonitor";
    system( "mkdir -p '$slavepath'" ) unless -d $slavepath;
    system "date > '$slavepath/$myname'";

    while(1)
    {
        warn "do ...\n";
        my $t = time;

        my $time = POSIX::strftime( "%Y-%m-%d %H:%M:%S", localtime( $t ) );

        my @ps = `ps -eo cmd`;
        chomp @ps;

        my %task = ( job_worker_task => 0, jobx_worker_task => 0 );
        my %proc = map{ $_ => 0 }keys %c;

        map{
            $_ = 'job_webshell' if $_ eq 'tty.js';
            if( defined $proc{$_} )
            {
                $proc{$_} ++;
            }elsif( $_ =~ /^(job[x]{0,1}_worker_task)_[a-zA-Z0-9]{12}$/ )
            {
                $task{$1} ++;
            }
        }@ps;

        my $ins = "replace into monitor (`time`,`time_s`,`stat`,`host`,`type`,`key`,`val`)value('%s','%s','%s','%s','%s','%s','%s')";
        eval{
            map{ $db->execute( sprintf $ins, $time, $t, $proc{$_} eq $c{$_} ? 'ok' : "err:$c{$_}", $myname, 'proc','openc3_'.$_, $proc{$_} ); }keys %proc;
            map{ $db->execute( sprintf $ins, $time, $t, $task{$_} > 100 ? 'danger' : $task{$_} > 10 ? 'warning' : 'healthy' , $myname, 'task','openc3_'.$_, $task{$_} ); }keys %task;
        };


        my %slavefile = %slave;
        map{ delete $slavefile{$_} if -f "$slavepath/$_"; }keys %slave;

        $db->execute( sprintf $ins, $time, $t, %slavefile ? sprintf( "err:%s", scalar keys %slavefile ) : 'ok' , $myname, 'filesys', 'openc3_glusterfs', scalar keys %slave );


        my $getNginxStatus = sub{

            my ( $url, $Host ) = @_;
            my $ua = LWP::UserAgent->new;
            $ua->default_header( Host => $Host ) if $Host;

            $ua->timeout( 3 );

            my $res = $ua->get( $url );
            if( $res->is_success && $res->content =~ /Active connections:\s*(\d+)\s*\n\s*server accepts handled requests\n\s*(\d+)\s+(\d+)\s+(\d+)\s*\n\s*Reading:\s*(\d+)\s*Writing:\s*(\d+)\s*Waiting:\s*(\d+)/ )
            {
                return ( 'ok', +{ active => $1, server => $2, accepts => $3, handled => $4, reading => $5, writing => $6, waiting => $7 } );
            }
            else
            {
                return ( 'err', +{ active => 0, server => 0, accepts => 0, handled => 0, reading => 0, writing => 0, waiting => 0 } );
            }
        };

        my @nginxcheck = (
            +{
                url => 'http://127.0.0.1:88/status/',
                Host => undef,
                name => 'api'
            },
            +{
                url => 'http://127.0.0.1/status/',
                Host => 'api.agent.open-c3.org',
                name => 'agent'
            },
            +{
                url => 'http://127.0.0.1/status/',
                Host => 'api.ci.open-c3.org',
                name => 'ci'
            },
            +{
                url => 'http://127.0.0.1/status/',
                Host => 'api.job.open-c3.org',
                name => 'job'
            },
            +{
                url => 'http://127.0.0.1/status/',
                Host => 'api.jobx.open-c3.org',
                name => 'jobx'
            },
            +{
                url => 'http://127.0.0.1/status/',
                Host => 'api.connector.open-c3.org',
                name => 'connector'
            },
        );

        for my $nginx ( @nginxcheck )
        {
            my ( $stat, $data ) = &$getNginxStatus($nginx->{url}, $nginx->{Host});
            $db->execute( sprintf $ins, $time, $t, $stat, $myname, 'nginx',"openc3_$nginx->{name}_status", $stat eq 'ok' ? 1 : 0 );
            map{ $db->execute( sprintf $ins, $time, $t, $stat, $myname, 'nginx',"openc3_$nginx->{name}_$_", $data->{$_} ); }keys %$data;
        }

        my $getMetrics = sub{

            my ( $url, $Host ) = @_;
            my $ua = LWP::UserAgent->new;
            $ua->default_header( Host => $Host ) if $Host;

            $ua->timeout( 3 );

            my $res = $ua->get( $url );

            if( $res->is_success )
            {
                my $data = eval{ JSON::from_json $res->content };
                return +{} if $@;
                return $data->{data};
            }
            else
            {
                return +{};
            }
        };

        my @mysqlcheck = (
            +{
                Host => 'api.agent.open-c3.org',
                name => 'agent'
            },
            +{
                Host => 'api.ci.open-c3.org',
                name => 'ci'
            },
            +{
                Host => 'api.job.open-c3.org',
                name => 'job'
            },
            +{
                Host => 'api.jobx.open-c3.org',
                name => 'jobx'
            },
            +{
                Host => 'api.connector.open-c3.org',
                name => 'connector'
            },
        );

        for my $mysql ( @mysqlcheck )
        {
            my $data = &$getMetrics( 'http://127.0.0.1/monitor/metrics/mysql', $mysql->{Host});
            map{ $db->execute( sprintf $ins, $time, $t, 'ok', 'openc3', 'mysql',"openc3_$mysql->{name}_mysql_$_", $data->{$_} ); }keys %$data;
        }

        my $cimetrics = &$getMetrics( 'http://127.0.0.1/monitor/metrics/app', 'api.ci.open-c3.org');
        map{ $db->execute( sprintf $ins, $time, $t, 'ok', 'openc3', 'app',"openc3_ci_app_$_", $cimetrics->{$_} ); }keys %$cimetrics;


        my $jobxmetrics = &$getMetrics( 'http://127.0.0.1/monitor/metrics/app', 'api.jobx.open-c3.org');
        map{ $db->execute( sprintf $ins, $time, $t, 'ok', 'openc3', 'app',"openc3_jobx_app_$_", $jobxmetrics->{$_} ); }keys %$jobxmetrics;


        my $jobmetrics = &$getMetrics( 'http://127.0.0.1/monitor/metrics/app', 'api.job.open-c3.org');
        map{ $db->execute( sprintf $ins, $time, $t, 'ok', 'openc3', 'app',"openc3_job_app_$_", $jobmetrics->{$_} ); }keys %$jobmetrics;

        my $agentmetrics = &$getMetrics( 'http://127.0.0.1/monitor/metrics/app', 'api.agent.open-c3.org');
        map{ $db->execute( sprintf $ins, $time, $t, 'ok', 'openc3', 'app',"openc3_agent_app_$_", $agentmetrics->{$_} ); }keys %$agentmetrics;


        my %diskalias = ( '/' => 'root', '/data' => 'data', '/data/glusterfs' => 'glusterfs' );
        my %disk = ( '/' => 0, '/data' => 0, '/data/glusterfs' => 0 );
        die "exec df fail.\n" unless my @df = MYDan::Collector::Util::qx( "LANG=en df -l 2>/dev/null" );
        for my $df ( map { [ ( split /\s+/, $_, 7 )[ 5, 4 ] ] } @df )
        {
            next unless defined $disk{$df->[0]};
            $df->[1] =~ s/%//;
            $disk{$df->[0]} = $df->[1];
        }

        map{ $db->execute( sprintf $ins, $time, $t, $disk{$_} > 90 ? 'err' : 'ok'  , $myname, 'os',"openc3_os_disk_$diskalias{$_}", $disk{$_} ); }keys %disk;


        my $load = 0;
        if( `cat /proc/loadavg` =~ /^([\d\.]+)\s+/ )
        {
            $load = $1;
        }
        
        $db->execute( sprintf $ins, $time, $t, $load > 20 ? 'err' : 'ok'  , $myname, 'os',"openc3_os_load", $load );

        my $gtime = POSIX::strftime( "%d_%H:%M", localtime( time - 60 ) );

        my $failcount = `tail -n 10000 /var/log/open-c3.log|egrep ": FAIL |: ERROR "|grep -E "$gtime"|wc -l`;
        chomp $failcount;
        $failcount = 110 unless $failcount =~ /^\d+$/;

        $db->execute( sprintf $ins, $time, $t, $failcount ? 'err' : 'ok'  , $myname, 'system',"openc3_system_fail", $failcount );

        my $usetime = int( time - $t );
        $db->execute( sprintf $ins, $time, $t, $usetime > 60 ? 'err' : 'ok'  , $myname, 'monitor',"openc3_monitor_usetime", $usetime );

        my $due = time + 60 - $t;
        sleep $due if $due > 0;
    }
}
